package server;

import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import models.Domain;
import models.Inlink;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Objectify;
import com.googlecode.objectify.ObjectifyService;
import com.googlecode.objectify.Query;

import controller.Utils;

public class Update {

	private URLFetchService fetcher = URLFetchServiceFactory
			.getURLFetchService();

	private Domain domain = new Domain();

	private TreeMap<String, String> check = new TreeMap<String, String>();

	private Future fDomain = null;

	private Future fWhois = null;

	private Future fIpAddress = null;

	private Future fAlexa = null;

	private Future fGoogleRank = null;

	private Future fAgeDomain = null;

	private Future fValidator = null;

	private Future fGoogleService = null;

	private Future fHeader = null;

	private Future fDns = null;

	private Future fTrace = null;

	private String content = "";

	private String alexaContent = "";

	public Update(Domain domain) {
		this.domain = domain;
		// Load content from urls
		loadUrlDomain();

		loadUrlWhois();

		loadUrlIpAddress();

		loadUrlAlexa();

		loadUrlGoogleRank();

		loadUrlAgeDomain();

		loadHeader();

		loadDns();

		loadTrace();

		// Get content form url loaded
		fDomainService();

		getWhois();

		getIpAddress();

		fAlexaService();

		getGooglePageRank();

		getAgeDomain();

		getHeader();

		getDns();

		getTrace();

		try {
			ObjectifyService.register(Domain.class);
		} catch (Exception e) {
			// TODO: handle exception
		}

		Objectify objectify = ObjectifyService.begin();
		System.out.println(this.domain.getOtherDomain().length());
		if (this.domain.getOtherDomain().length() < 10) {
			System.out.println("a");
			int ran = new Random().nextInt(1000000);
			if (this.domain.getAlexa() != 0) {
				ran = this.domain.getAlexa();
			}

			Query<Domain> list = objectify.query(Domain.class)
					.filter("alexa >", ran).limit(5);
			String otherDomain = "";
			for (Domain ar : list) {
				otherDomain += ar.getDomain() + ",";
			}
			list = objectify.query(Domain.class).order("-alexa")
					.filter("alexa <", ran).limit(5);
			for (Domain ar : list) {
				otherDomain += ar.getDomain() + ",";
			}
			otherDomain = otherDomain.replaceAll("[,]+$", "");
			System.out.println(otherDomain);
			this.domain.setOtherDomain(otherDomain);
		}
		String start = "";
		try {
			start = Utils.returnSubDomainTo(this.domain.getDomain()).substring(
					0, 1);
		} catch (Exception e) {
			// TODO: handle exception
		}
		this.domain.setStart(start);
		this.domain.setDate(new Date());
		PutCache.putCacheList(this.domain.getDomain(), "updates", 19);

		objectify.put(this.domain);
	}

	/************************************* Load URL *************************************/

	private void loadHeader() {
		try {
			URL urlHeader = new URL(
					"http://network-tools.com/default.asp?prog=httphead&host="
							+ domain.getDomain());
			fHeader = fetcher.fetchAsync(urlHeader);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fDomain", "Can't run fDomain");
		}
	}

	private void loadDns() {
		try {
			URL urlDns = new URL(
					"http://network-tools.com/default.asp?prog=dnsrec&host="
							+ domain.getDomain());
			fDns = fetcher.fetchAsync(urlDns);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fDomain", "Can't run fDomain");
		}
	}

	private void loadTrace() {
		try {
			URL urlTrace = new URL(
					"http://network-tools.com/default.asp?prog=trace&host="
							+ domain.getDomain());
			fTrace = fetcher.fetchAsync(urlTrace);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fDomain", "Can't run fDomain");
		}
	}

	private void loadUrlDomain() {
		try {
			URL urlDomain = new URL("http://" + domain.getDomain());
			fDomain = fetcher.fetchAsync(urlDomain);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fDomain", "Can't run fDomain");
		}
	}

	private void loadUrlWhois() {
		try {
			ArrayList<String> arr = new ArrayList<String>();
			arr.add("http://yoo-ports.appspot.com/whois?name="+domain.getDomain());
			arr.add("http://www.tonghoptin.com//whois?name="+domain.getDomain());
			arr.add("http://yoospain.appspot.com/whois?name="+domain.getDomain());
			arr.add("http://www.as7.org/whoisview?name="+domain.getDomain());
			arr.add("http://www.openbos.com/whoisview?name="+domain.getDomain());
			Random random = new Random();
            URL urlWhois = new URL(arr.get(random.nextInt(4)));
            fWhois = fetcher.fetchAsync(urlWhois);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Can't run fWhois");
        }
	}

	private void loadUrlIpAddress() {
		try {
			URL urlIpAddress = new URL(
					"http://www.nearby.org.uk/api/dig.php?l="
							+ domain.getDomain());
			fIpAddress = fetcher.fetchAsync(urlIpAddress);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fIpAddress", "Can't run fIpAddress");
		}
	}

	private void loadUrlAlexa() {
		try {
			URL urlAlexa = new URL("http://www.alexa.com/siteinfo/"
					+ domain.getDomain());
			fAlexa = fetcher.fetchAsync(urlAlexa);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fAlexa", "Can't run fAlexa");
		}
	}

	private void loadUrlGoogleRank() {
		try {
			URL urlGoogleRank = new URL("http://gpageranks.appspot.com/?url="
					+ domain.getDomain());
			fGoogleRank = fetcher.fetchAsync(urlGoogleRank);
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fGoogleRank", "Can't run fGoogleRank");
		}
	}

	private void loadUrlAgeDomain() {
		try {
			URL urlAgeDomain = new URL(
					"http://www.seologs.com/dns/domain-check.html");
			HTTPRequest request = new HTTPRequest(urlAgeDomain, HTTPMethod.POST);
			request.setHeader(new HTTPHeader("Content-Type",
					"application/x-www-form-urlencoded"));
			String params = "query=" + domain.getDomain() + "&output=cdt";
			request.setPayload(params.getBytes());
			fAgeDomain = fetcher.fetchAsync(request);

		} catch (Exception e) {
			e.printStackTrace();
			check.put("fAgeDomain", "Can't run fAgeDomain");
		}
	}

	/************************************* Get content *************************************/
	private void fDomainService() {
		try {
			HTTPResponse response = (HTTPResponse) fDomain.get();

			// if redirects are followed, this returns the final URL we are
			// redirected to
			URL finalUrl = response.getFinalUrl();

			String charset = "utf-8";

			this.content = new String(response.getContent(), charset);

			Document doc = Jsoup.parse(this.content);

			int index = doc.select("meta").toString().toLowerCase()
					.indexOf("charset=");
			if (index != -1) {
				charset = doc.select("meta").toString().substring(index + 8)
						.replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "")
						.replaceAll("[\\'\\\"]+", "");
			}

			this.content = new String(response.getContent(), charset);

			getServerInfo(response);

			getSiteInfo();
			getGoogleAnalytic();
			getGoogleAdsense();
			getJavascriptLibrary();
			getInlink();

		} catch (Exception e) {
			e.printStackTrace();
			check.put("fDomain", "Can't response fDomain");
		}
	}

	private void fAlexaService() {
		try {
			HTTPResponse response = (HTTPResponse) fAlexa.get();

			this.alexaContent = new String(response.getContent(), "utf-8");

			getAlexaRank();
			getPopularKeyword();

		} catch (Exception e) {
			e.printStackTrace();
			check.put("fAlexa", "Can't response fAlexa");
		}
	}

	private void getHeader() {
		try {
			HTTPResponse response = (HTTPResponse) fHeader.get();

			Document doc = Jsoup.parse(new String(response.getContent(),
					"utf-8"));
			
			Element elements = doc.select("div[style=width:680px; float:left; margin-top:25px;]").get(0);
			elements.select("center").remove();
			
			this.domain.setHeader(new Text(elements.html()));
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fAlexa", "Can't response fAlexa");
		}
	}

	private void getDns() {
		try {
			HTTPResponse response = (HTTPResponse) fDns.get();

			Document doc = Jsoup.parse(new String(response.getContent(),
					"utf-8"));
			Element elements = doc.select("div[style=width:680px; float:left; margin-top:25px;]").get(0);
			elements.select("center").remove();
			this.domain.setDns(new Text(elements.html()));
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fAlexa", "Can't response fAlexa");
		}
	}

	private void getTrace() {
		try {
			HTTPResponse response = (HTTPResponse) fTrace.get();

			Document doc = Jsoup.parse(new String(response.getContent(),
					"utf-8"));
			Element elements = doc.select("div[style=width:680px; float:left; margin-top:25px;]").get(0);
			elements.select("center").remove();
			this.domain.setTrace(new Text(elements.html()));
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fAlexa", "Can't response fAlexa");
		}
	}

	private void getPopularKeyword() {
		try {
			Document doc = Jsoup.parse(this.alexaContent);
			Elements elements = doc.select("#top-keywords-from-search");
			String keyword = "";
			if (elements.size() > 0) {
				for (Element element : elements.first().select("u")) {
					keyword += element.text() + ", ";
				}
			}
			domain.setPopularKeyword(new Text(keyword));
		} catch (Exception e) {
			e.printStackTrace();
			check.put("alexaRank", "Can't get position alexa rank of domain "
					+ domain.getDomain());
		}
	}

	private void getServerInfo(HTTPResponse response) {
		try {
			// Get webServer
			String webServer = "";
			String programmingLanguage = "";
			for (int i = 0; i < response.getHeaders().size(); i++) {
				if (response.getHeaders().get(i).getName()
						.equalsIgnoreCase("server")) {
					webServer = response.getHeaders().get(i).getValue();
				}

				if (response.getHeaders().get(i).getName()
						.equalsIgnoreCase("x-powered-by")) {
					programmingLanguage = response.getHeaders().get(i)
							.getValue();
				}
				// System.out.println(response.getHeaders().get(i).getName() +
				// " - " + response.getHeaders().get(i).getValue());
			}

			this.domain.setServer(webServer);
			this.domain.setLanguage(programmingLanguage);

		} catch (Exception e) {
			this.check.put("server", domain.getDomain()
					+ " - Webserver not found");
		}
	}

	private void getSiteInfo() {
		try {
			Document doc = Jsoup.parse(this.content);

			String title = "";
			String description = "";
			String keywords = "";
			String des = "";
			try {
				title = doc.select("title").first().text();
			} catch (Exception e) {
				e.printStackTrace();
				this.check.put("title", "Can't get info title");
			}

			try {
				keywords = doc.select("meta[name=keywords]").first()
						.attr("content");
			} catch (Exception e) {
				try {
					keywords = doc.select("meta[http-equiv=keywords]").first()
							.attr("content");
				} catch (Exception e2) {
					this.check.put("keywords", "Can't get info keywords");
				}
			}

			try {
				description = doc.select("meta[name=description]").first()
						.attr("content");
			} catch (Exception e) {
				try {
					description = doc.select("meta[http-equiv=description]")
							.first().attr("content");
				} catch (Exception e2) {
					if (doc.select("body").first().text().length() > 260) {
						description = doc.select("body").first().text()
								.substring(0, 250)
								.replaceAll("[ ]+[\\w\\W]{0,10}+$", "")
								+ " ...";
					} else {
						description = doc.select("body").first().text();
					}
				}
			}
			des = doc.select("body p").text();
			if (des.length() < 100)
				des = doc.select("body").first().text();
			if (des.length() > 250)
				des = des.substring(0, 240);

			if (description.length() > 260) {
				description = description.substring(0, 250);
			}

			this.domain.setTitle(title);
			this.domain.setKeywords(new Text(keywords));
			this.domain.setDescription(new Text(description));
			this.domain.setDes(new Text(des));

		} catch (Exception e) {
			e.printStackTrace();
			this.check.put("siteinfo",
					"Can't connect to domain " + domain.getDomain());
		}
	}

	private void getGoogleAnalytic() {
		int begin = content.indexOf("_gaq.push(['_setAccount', '");
		String analyticAccount = "";
		if (begin > 0) {
			begin += "_gaq.push(['_setAccount', '".length();
			int end = content.indexOf("'", begin);
			analyticAccount = content.substring(begin, end);
			begin = analyticAccount.lastIndexOf("-") - 1;

			analyticAccount = analyticAccount.substring(0, begin);
		} else {
			begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
			if (begin > 0) {
				begin += "_gaq.push([\"_setAccount\", \"".length();
				int end = content.indexOf("\"", begin);
				analyticAccount = content.substring(begin, end);

			} else {
				begin = content.indexOf("_uacct");
				if (begin > 0) {
					begin += "_uacct".length();
					analyticAccount = content.substring(begin, begin + 30)
							.replaceAll("^[ =\\'\\\"]+", "")
							.replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
					begin = analyticAccount.lastIndexOf("-");

					analyticAccount = analyticAccount.substring(0, begin);
				} else {
					Pattern p = Pattern.compile("UA-[0-9]+");

					Matcher m = p.matcher(content);
					while (m.find()) {
						analyticAccount = m.group();
					}
				}
			}
		}

		this.domain.setAnalytic(analyticAccount);
	}

	private void getGoogleAdsense() {
		int begin = content.indexOf("google_ad_client = \"");
		String adsAccount = "";
		if (begin > 0) {
			begin += "google_ad_client = \"', '".length();
			int end = content.indexOf("\"", begin);
			adsAccount = content.substring(begin, end);
		}

		this.domain.setAdsense(adsAccount);
	}

	private void getJavascriptLibrary() {
		List<String> jsLib = new ArrayList<String>();
		if (content.indexOf("jquery") > 0)
			jsLib.add("jquery");

		if (content.indexOf("prototype.js") > 0)
			jsLib.add("prototype");

		if (content.indexOf("extjs") > 0)
			jsLib.add("extjs");

		if (content.indexOf("yui") > 0)
			jsLib.add("yui");

		if (content.indexOf("dojo") > 0)
			jsLib.add("dojo");

		if (content.indexOf("gwt") > 0)
			jsLib.add("gwt");

		if (content.indexOf("easeljs") > 0)
			jsLib.add("easeljs");

		this.domain.setJs(StringHelper.listToString(jsLib, ", "));
	}

	private void getWhois() {
		try {

			HTTPResponse response = (HTTPResponse) fWhois.get();
            
            Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
            
            this.domain.setWhois(new Text(doc.select("#whois-p1").html()));
		} catch (Exception e) {
			e.printStackTrace();
			check.put("fWhois", "Error when get value whois");
		}
	}

	private void getIpAddress() {
		try {
			HTTPResponse response = (HTTPResponse) fIpAddress.get();

			this.domain.setIp(new String(response.getContent()));
		} catch (Exception e) {
			e.printStackTrace();
			check.put("ipAddress", "Can't get ipAddress for this domain "
					+ domain.getDomain());
		}
	}

	private void getAlexaRank() {
		try {
			Document doc = Jsoup.parse(this.alexaContent);
			String strAlexaRank = doc.select("#siteStats")
					.select("tbody tr td div").first().text().trim()
					.replace(",", "");
			int alexaRank = 0;

			try {
				if (strAlexaRank != null && !strAlexaRank.isEmpty()) {
					alexaRank = Integer.parseInt(strAlexaRank.replaceAll(
							"[^0-9]+", ""));
				}
			} catch (Exception e) {
				e.printStackTrace();
				check.put(
						"alexaGlobeRank",
						"Can't get position alexa rank of domain "
								+ domain.getDomain());
			}

			this.domain.setAlexa(alexaRank);

		} catch (Exception e) {
			e.printStackTrace();
			check.put("alexaRank", "Can't get position alexa rank of domain "
					+ domain.getDomain());
		}
	}

	private void getGooglePageRank() {
		try {
			HTTPResponse response = (HTTPResponse) fGoogleRank.get();

			Document doc = Jsoup.parse(new String(response.getContent()));
			String googlePageRank = "0";
			String google = doc.select(".show").select("span").first()
					.select("font").text().replaceAll("[^0-9]", "");
			try {
				if (google != null && !google.isEmpty()) {
					googlePageRank = google;
				}
			} catch (Exception e) {
				e.printStackTrace();
				check.put("googlePageRank",
						"Can't get position google page rank of domain "
								+ domain.getDomain());
			}

			this.domain.setGoogle(Integer.parseInt(googlePageRank.replaceAll(
					"[^0-9]+", "")));

		} catch (Exception e) {
			e.printStackTrace();
			check.put(
					"googlePageRank",
					"Can't get position google page rank of domain "
							+ domain.getDomain());
		}
	}

	private void getAgeDomain() {
		try {
			HTTPResponse response = (HTTPResponse) fAgeDomain.get();

			Document doc = Jsoup.parse(new String(response.getContent()));
			String age = doc.select("div[class=KonaBody]").select("p").text();
			String[] ageSplit = age.split("(approximately)");

			if (ageSplit.length == 2) {
				this.domain.setAge(ageSplit[1].replaceAll("[):]+", ""));
			}

		} catch (Exception e) {
			e.printStackTrace();
			check.put("age", "Can't get age of domain " + domain.getDomain());
		}
	}

	private void getInlink() {
		try {
			Document doc = Jsoup.parse(this.content);
			Elements link = doc.select("a");

			ArrayList<String> listLink = new ArrayList<String>();
			String check = null;

			try {
				ObjectifyService.register(Inlink.class);
			} catch (Exception e) {
				// TODO: handle exception
			}
			Objectify objectify = ObjectifyService.begin();

			for (int i = 0; i < link.size(); i++) {
				check = link.get(i).attr("rel");
				if (Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href"))) {
					if (check != null && !check.equals("nofollow")
							|| check == null) {
						Pattern p = Pattern
								.compile("^http[^\\/]+[\\/]{2}[^\\/]+");

						Matcher m = p.matcher(link.get(i).attr("href"));
						while (m.find()) {
							if (!listLink.contains(m.group())
									&& !Utils.fomatDomain(m.group()).equals(
											this.domain.getDomain())) {
								if (Utils.checkDomain(Utils.fomatDomain(m
										.group()))) {
									listLink.add(m.group());

									try {
										Inlink inlink = objectify
												.get(new Key<Inlink>(
														Inlink.class,
														Utils.fomatDomain(m
																.group())));
										inlink.addInLink(this.domain
												.getDomain());
										objectify.put(inlink);
									} catch (Exception e) {
										// e.printStackTrace();
										Inlink inlink = new Inlink();
										inlink.setDomain(Utils.fomatDomain(m
												.group()));
										inlink.addInLink(this.domain
												.getDomain());
										objectify.put(inlink);
									}
								}
							}
						}
					}
				}
			}

			this.domain.setOutLink(listLink);

		} catch (Exception e) {
			e.printStackTrace();
			check.put("inlink",
					"Can't get inlink of domain " + domain.getDomain());
		}
	}
}
